mbox-sync.c revision a205d315b0978985ba77d871f44e4a98273612e6
5f5870385cff47efd2f58e7892f251cf13761528Timo Sirainen/* Copyright (C) 2004 Timo Sirainen */
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen/*
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen Modifying mbox can be slow, so we try to do it all at once minimizing the
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen required disk I/O. We may need to:
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen - Update message flags in Status, X-Status and X-Keywords headers
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen - Write missing X-UID and X-IMAPbase headers
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen - Write missing or broken Content-Length header if there's space
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen - Expunge specified messages
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
7a7d2aa11e46195e2d92d6c337d7e78052a5ce67Timo Sirainen Here's how we do it:
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen - Start reading the mails mail headers from the beginning
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen - X-Keywords and X-UID headers may contain extra spaces at the end of them,
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen remember how much extra each message has and offset to beginning of the
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen spaces
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen - If message flags are dirty and there's enough space to write them, do it
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen - If we didn't have enough space, remember how much was missing and keep
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen the total amount of them
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen - When we encounter expunged message, check if the amount of empty space in
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen previous messages plus size of expunged message is enough to cover the
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen missing space. If yes,
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen - execute the rewrite plan
499fec3443374cc89fb8c83b8027c1614097d7a3Timo Sirainen - forget all the messages before the expunged message. only remember
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen how much data we still have to move to cover the expunged message
f5a7396b31762a1f876517e13ce9065820139f7cTimo Sirainen - If we encounter end of file, grow the file and execute the rewrite plan
f5a7396b31762a1f876517e13ce9065820139f7cTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen Rewrite plan goes:
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen - Start from the first message that needs more space
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen - If there's expunged messages before us, we have to write over them.
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen - Move all messages after it backwards to fill it
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen - Each moved message's X-Keywords header should have n bytes extra
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen space, unless there's not enough space to do it.
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen - If there's no expunged messages, we can move data either forward or
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen backward to get it. Calculate which requires less moving. Forward
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen counting may encounter more messages which require extra space, count
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen that too.
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen - If we decide to move forwards and we had to go through dirty
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen messages, do the moving from last to first dirty message
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen - If we encounter end of file, grow the file enough to get the required
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen amount of space plus enough space to fill X-Keywords headers full of
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen spaces.
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen*/
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen#include "lib.h"
942302b0247403645394d848b3c620ead262a2a5Timo Sirainen#include "ioloop.h"
942302b0247403645394d848b3c620ead262a2a5Timo Sirainen#include "buffer.h"
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen#include "istream.h"
942302b0247403645394d848b3c620ead262a2a5Timo Sirainen#include "file-set-size.h"
942302b0247403645394d848b3c620ead262a2a5Timo Sirainen#include "str.h"
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen#include "write-full.h"
11352dc3e4b29f3d2763c82f8ea4f99e8daf4fa3Timo Sirainen#include "istream-raw-mbox.h"
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen#include "mbox-storage.h"
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen#include "mbox-file.h"
942302b0247403645394d848b3c620ead262a2a5Timo Sirainen#include "mbox-lock.h"
942302b0247403645394d848b3c620ead262a2a5Timo Sirainen#include "mbox-sync-private.h"
942302b0247403645394d848b3c620ead262a2a5Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen#include <stddef.h>
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen#include <sys/stat.h>
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen#define MBOX_SYNC_SECS 1
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
cd8507179823de33d6e8242e10dbc15d136245b5Timo Sirainen/* returns -1 = error, 0 = mbox changed since previous lock, 1 = didn't */
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainenstatic int mbox_sync_lock(struct mbox_sync_context *sync_ctx, int lock_type)
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen{
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen struct index_mailbox *ibox = sync_ctx->ibox;
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen struct stat old_st, st;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen uoff_t old_from_offset = 0, old_offset = 0;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen i_assert(lock_type != F_WRLCK || !ibox->mbox_readonly);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen if (sync_ctx->lock_id != 0) {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen if (fstat(sync_ctx->fd, &old_st) < 0) {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen mbox_set_syscall_error(ibox, "stat()");
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen return -1;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen }
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen old_from_offset =
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen istream_raw_mbox_get_start_offset(sync_ctx->input);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen old_offset = sync_ctx->input->v_offset;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen (void)mbox_unlock(ibox, sync_ctx->lock_id);
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen sync_ctx->lock_id = 0;
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen } else {
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen memset(&old_st, 0, sizeof(old_st));
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen }
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen if (mbox_lock(ibox, lock_type, &sync_ctx->lock_id) <= 0)
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen return -1;
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen if (mbox_file_open_stream(ibox) < 0)
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen return -1;
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen sync_ctx->file_input = sync_ctx->ibox->mbox_file_stream;
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen sync_ctx->input = sync_ctx->ibox->mbox_stream;
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen sync_ctx->fd = sync_ctx->ibox->mbox_fd;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if (old_st.st_mtime == 0) {
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen /* we didn't have the file open before -> it changed */
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen return 0;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen }
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if (fstat(sync_ctx->fd, &st) < 0) {
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen mbox_set_syscall_error(ibox, "fstat()");
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen return -1;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen }
4ece61edd7c266a4b8f3b290a7f0a3cb3d13ca0fTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if (st.st_mtime != old_st.st_mtime || st.st_size != old_st.st_size ||
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen st.st_ino != old_st.st_ino ||
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen !CMP_DEV_T(st.st_dev, old_st.st_dev) ||
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen time(NULL) - st.st_mtime <= MBOX_SYNC_SECS)
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen return 0;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen /* same as before. we'll have to fix mbox stream to contain
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen correct from_offset, hdr_offset and body_offset. so, seek
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen to from_offset and read through the header. */
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen if (istream_raw_mbox_seek(sync_ctx->input, old_from_offset) < 0) {
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen mail_storage_set_critical(ibox->box.storage,
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen "Message offset %s changed unexpectedly for mbox file "
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen "%s", dec2str(old_from_offset), sync_ctx->ibox->path);
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen return 0;
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen }
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen (void)istream_raw_mbox_get_body_offset(sync_ctx->input);
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen i_stream_seek(sync_ctx->input, old_offset);
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen return 1;
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen}
c93ff0433cc3d348116f75a64f9988fedb86fd18Timo Sirainen
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainenint mbox_sync_seek(struct mbox_sync_context *sync_ctx, uoff_t from_offset)
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen{
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen if (istream_raw_mbox_seek(sync_ctx->input, from_offset) < 0) {
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen mail_storage_set_critical(sync_ctx->ibox->box.storage,
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen "Unexpectedly lost From-line at offset %"PRIuUOFF_T
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen " from mbox file %s", from_offset,
11352dc3e4b29f3d2763c82f8ea4f99e8daf4fa3Timo Sirainen sync_ctx->ibox->path);
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen return -1;
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen }
755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainen return 0;
755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainen}
755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainen
755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainenstatic int mbox_sync_grow_file(struct mbox_sync_context *sync_ctx,
755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainen struct mbox_sync_mail_context *mail_ctx,
755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainen uoff_t grow_size)
755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainen{
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen uoff_t src_offset, file_size;
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen i_assert(grow_size > 0);
755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainen
755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainen /* put the extra space between last message's header and body */
755aea84bbe2b15ed7fe991f6462a93333ff571fTimo Sirainen file_size = i_stream_get_size(sync_ctx->file_input) + grow_size;
8305127d1074cf9a1e25dec9be2735276462079dTimo Sirainen if (file_set_size(sync_ctx->fd, file_size) < 0) {
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen mbox_set_syscall_error(sync_ctx->ibox, "file_set_size()");
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen return -1;
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen }
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen src_offset = mail_ctx->body_offset;
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen mail_ctx->body_offset += grow_size;
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen if (mbox_move(sync_ctx, mail_ctx->body_offset, src_offset,
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen file_size - mail_ctx->body_offset) < 0)
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen return -1;
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen istream_raw_mbox_flush(sync_ctx->input);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen return 0;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen}
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainenstatic void mbox_sync_buffer_delete_old(buffer_t *syncs_buf, uint32_t uid)
596433ccbca59ce2328dc1d029586154cd937155Timo Sirainen{
596433ccbca59ce2328dc1d029586154cd937155Timo Sirainen struct mail_index_sync_rec *sync;
4145cbac82bfc0c8bfeceeca0ef841700117930cTimo Sirainen size_t size, src, dest;
4145cbac82bfc0c8bfeceeca0ef841700117930cTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen sync = buffer_get_modifyable_data(syncs_buf, &size);
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen size /= sizeof(*sync);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
e5fd6dfd0a492e4708d4dbb7971d7fc5d7b8fd85Timo Sirainen for (src = dest = 0; src < size; src++) {
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen if (sync[src].uid2 >= uid) {
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen if (src != dest)
132487b9a47c2eb6fc80cfa2b0aaf82c6dc3af56Timo Sirainen sync[dest] = sync[src];
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen dest++;
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen }
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen }
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen buffer_set_used_size(syncs_buf, dest * sizeof(*sync));
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen}
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainenstatic int
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainenmbox_sync_read_next_mail(struct mbox_sync_context *sync_ctx,
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen struct mbox_sync_mail_context *mail_ctx)
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen{
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen /* get EOF */
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen (void)istream_raw_mbox_get_header_offset(sync_ctx->input);
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if (istream_raw_mbox_is_eof(sync_ctx->input))
7f3b826a89bcb7a72759912e99f574b28309fe1bTimo Sirainen return 0;
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen memset(mail_ctx, 0, sizeof(*mail_ctx));
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen mail_ctx->sync_ctx = sync_ctx;
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen mail_ctx->seq = ++sync_ctx->seq;
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen mail_ctx->header = sync_ctx->header;
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen mail_ctx->from_offset =
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen istream_raw_mbox_get_start_offset(sync_ctx->input);
6b399f555c9c5c722d4cd5eab8faa02b2a4731d3Timo Sirainen mail_ctx->mail.offset =
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen istream_raw_mbox_get_header_offset(sync_ctx->input);
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if (mail_ctx->seq == 1)
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen sync_ctx->seen_first_mail = TRUE;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if (mail_ctx->seq > 1 && sync_ctx->dest_first_mail) {
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen /* First message was expunged and this is the next one.
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen Skip \n header */
e10d8b1291090c26b9ef499637e6e632485ca5beTimo Sirainen mail_ctx->from_offset++;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen }
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx, FALSE);
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen i_assert(sync_ctx->input->v_offset != mail_ctx->from_offset);
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen mail_ctx->mail.body_size =
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen istream_raw_mbox_get_body_size(sync_ctx->input,
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen mail_ctx->content_length);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen i_assert(mail_ctx->mail.body_size < OFF_T_MAX);
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
18ccd19c244f49665fe03cda785efa066d2c38dfTimo Sirainen /* save the offset permanently with recent flag state */
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen mail_ctx->mail.from_offset = mail_ctx->from_offset;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if ((mail_ctx->mail.flags & MBOX_NONRECENT) == 0) {
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen /* need to add 'O' flag to Status-header */
c529313e1cbc22244d4528e80aa3e485f8806cd3Timo Sirainen mail_ctx->need_rewrite = TRUE;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen // FIXME: save it somewhere
d22301419109ed4a38351715e6760011421dadecTimo Sirainen }
d22301419109ed4a38351715e6760011421dadecTimo Sirainen return 1;
d22301419109ed4a38351715e6760011421dadecTimo Sirainen}
d22301419109ed4a38351715e6760011421dadecTimo Sirainen
d22301419109ed4a38351715e6760011421dadecTimo Sirainenstatic int mbox_sync_buf_have_expunges(buffer_t *syncs_buf)
499fec3443374cc89fb8c83b8027c1614097d7a3Timo Sirainen{
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen const struct mail_index_sync_rec *sync;
8c909e451d14075c05d90382cf8eebc4e354f569Timo Sirainen size_t size, i;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen sync = buffer_get_data(syncs_buf, &size);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen size /= sizeof(*sync);
d22301419109ed4a38351715e6760011421dadecTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen for (i = 0; i < size; i++) {
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if (sync[i].type == MAIL_INDEX_SYNC_TYPE_EXPUNGE)
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen return TRUE;
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen }
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen return FALSE;
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen}
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainenstatic int mbox_sync_read_index_syncs(struct mbox_sync_context *sync_ctx,
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen uint32_t uid, int *sync_expunge_r)
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen{
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen struct mail_index_sync_rec *sync_rec = &sync_ctx->sync_rec;
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen int ret;
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen *sync_expunge_r = FALSE;
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen if (sync_ctx->ibox->mbox_readonly || sync_ctx->index_sync_ctx == NULL)
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen return 0;
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen mbox_sync_buffer_delete_old(sync_ctx->syncs, uid);
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen while (uid >= sync_rec->uid1) {
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen if (sync_rec->uid1 != 0) {
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen i_assert(uid <= sync_rec->uid2);
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen buffer_append(sync_ctx->syncs, sync_rec,
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen sizeof(*sync_rec));
c4bb0320ab43ea35fa6df88fc745fdad906cee44Timo Sirainen
c4bb0320ab43ea35fa6df88fc745fdad906cee44Timo Sirainen if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE)
c4bb0320ab43ea35fa6df88fc745fdad906cee44Timo Sirainen *sync_expunge_r = TRUE;
047e3bbb00e68a0d43355e11a67b2e912e06de19Timo Sirainen }
c4bb0320ab43ea35fa6df88fc745fdad906cee44Timo Sirainen
047e3bbb00e68a0d43355e11a67b2e912e06de19Timo Sirainen ret = mail_index_sync_next(sync_ctx->index_sync_ctx, sync_rec);
047e3bbb00e68a0d43355e11a67b2e912e06de19Timo Sirainen if (ret < 0)
047e3bbb00e68a0d43355e11a67b2e912e06de19Timo Sirainen return -1;
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if (ret == 0) {
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen memset(sync_rec, 0, sizeof(*sync_rec));
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen break;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen }
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen }
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen if (!*sync_expunge_r)
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen *sync_expunge_r = mbox_sync_buf_have_expunges(sync_ctx->syncs);
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen return 0;
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen}
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainenstatic void mbox_sync_apply_index_syncs(buffer_t *syncs_buf, uint8_t *flags,
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen keywords_mask_t keywords)
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen{
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen const struct mail_index_sync_rec *sync;
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen size_t size, i;
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen sync = buffer_get_data(syncs_buf, &size);
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen size /= sizeof(*sync);
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen for (i = 0; i < size; i++) {
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (sync[i].type != MAIL_INDEX_SYNC_TYPE_FLAGS)
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen continue;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen mail_index_sync_flags_apply(&sync[i], flags, keywords);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen }
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen}
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainenstatic int
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainenmbox_sync_read_index_rec(struct mbox_sync_context *sync_ctx,
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen uint32_t uid, const struct mail_index_record **rec_r)
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen{
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen const struct mail_index_record *rec = NULL;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen uint32_t messages_count;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
942302b0247403645394d848b3c620ead262a2a5Timo Sirainen messages_count = mail_index_view_get_message_count(sync_ctx->sync_view);
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen while (sync_ctx->idx_seq < messages_count) {
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen if (mail_index_lookup(sync_ctx->sync_view,
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen ++sync_ctx->idx_seq, &rec) < 0) {
18ccd19c244f49665fe03cda785efa066d2c38dfTimo Sirainen mail_storage_set_index_error(sync_ctx->ibox);
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen return -1;
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen }
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen
db0735f9b388c5bcfb781b1b25015e898d63d953Timo Sirainen if (uid <= rec->uid)
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen break;
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen /* externally expunged message, remove from index */
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen rec = NULL;
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen }
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen if (rec != NULL && rec->uid != uid) {
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen /* new UID in the middle of the mailbox - shouldn't happen */
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen mail_storage_set_critical(sync_ctx->ibox->box.storage,
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen "mbox sync: UID inserted in the middle of mailbox %s "
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen "(%u > %u)", sync_ctx->ibox->path, rec->uid, uid);
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen mail_index_mark_corrupted(sync_ctx->ibox->index);
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen return -1;
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen }
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen *rec_r = rec;
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen return 0;
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen}
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainenstatic int mbox_sync_get_from_offset(struct mbox_sync_context *sync_ctx,
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen uint32_t seq, uint64_t *offset_r)
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen{
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen const void *data;
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen /* see if from_offset needs updating */
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen if (mail_index_lookup_extra(sync_ctx->sync_view, seq,
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen sync_ctx->ibox->mbox_extra_idx,
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen &data) < 0) {
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen mail_storage_set_index_error(sync_ctx->ibox);
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen return -1;
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen }
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen *offset_r = *((const uint64_t *)data);
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen return 0;
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen}
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainenstatic int
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainenmbox_sync_update_from_offset(struct mbox_sync_context *sync_ctx,
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen struct mbox_sync_mail *mail,
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen int nocheck)
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen{
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen uint64_t offset;
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen if (!nocheck) {
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen if (mbox_sync_get_from_offset(sync_ctx, sync_ctx->idx_seq,
d859478e8b106de6cea54f26861bd4232c92f62cTimo Sirainen &offset) < 0)
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen return -1;
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen if (offset == mail->from_offset)
d859478e8b106de6cea54f26861bd4232c92f62cTimo Sirainen return 0;
206ed2f6fa3a6fb291498627b2da626581c07a18Timo Sirainen } else {
206ed2f6fa3a6fb291498627b2da626581c07a18Timo Sirainen offset = mail->from_offset;
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen }
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen mail_index_update_extra_rec(sync_ctx->t, sync_ctx->idx_seq,
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen sync_ctx->ibox->mbox_extra_idx, &offset);
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen return 0;
206ed2f6fa3a6fb291498627b2da626581c07a18Timo Sirainen}
d859478e8b106de6cea54f26861bd4232c92f62cTimo Sirainen
d859478e8b106de6cea54f26861bd4232c92f62cTimo Sirainenstatic int mbox_sync_update_index(struct mbox_sync_context *sync_ctx,
77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen struct mbox_sync_mail *mail,
77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen const struct mail_index_record *rec)
77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen{
77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen keywords_mask_t idx_keywords;
77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen uint8_t idx_flags, mbox_flags;
77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen
77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen if (rec == NULL) {
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen /* new message */
77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen mail_index_append(sync_ctx->t, mail->uid, &sync_ctx->idx_seq);
77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen mbox_flags = mail->flags & (MAIL_FLAGS_MASK^MAIL_RECENT);
77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen MODIFY_REPLACE, mbox_flags,
77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen mail->keywords);
77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen } else {
77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen /* see if flags changed */
77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen idx_flags = rec->flags;
77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen memcpy(idx_keywords, rec->keywords, INDEX_KEYWORDS_BYTE_COUNT);
77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen mbox_sync_apply_index_syncs(sync_ctx->syncs,
9438ecaf1caee1bb33c8d7f638742875ac42c4e5Timo Sirainen &idx_flags, idx_keywords);
9438ecaf1caee1bb33c8d7f638742875ac42c4e5Timo Sirainen
77d8223da3da23b731257596abefa77e4485b77dTimo Sirainen mbox_flags = (rec->flags & ~MAIL_FLAGS_MASK) |
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen (mail->flags & (MAIL_FLAGS_MASK^MAIL_RECENT));
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (idx_flags != mbox_flags ||
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen memcmp(idx_keywords, mail->keywords,
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen INDEX_KEYWORDS_BYTE_COUNT) != 0) {
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen MODIFY_REPLACE, mbox_flags,
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen mail->keywords);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen }
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen }
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen /* update from_offsets, but not if we're going to rewrite this message.
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen rewriting would just move it anyway. */
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (sync_ctx->need_space_seq == 0) {
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen int nocheck = rec == NULL || sync_ctx->expunged_space > 0;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (mbox_sync_update_from_offset(sync_ctx, mail, nocheck) < 0)
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen return -1;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen }
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen return 0;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen}
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainenstatic int mbox_read_from_line(struct mbox_sync_mail_context *ctx)
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen{
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen struct istream *input = ctx->sync_ctx->file_input;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen const unsigned char *data;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen size_t size, from_line_size;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen buffer_set_used_size(ctx->sync_ctx->from_line, 0);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen from_line_size = ctx->hdr_offset - ctx->from_offset;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen i_stream_seek(input, ctx->from_offset);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen for (;;) {
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen data = i_stream_get_data(input, &size);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (size >= from_line_size)
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen size = from_line_size;
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen buffer_append(ctx->sync_ctx->from_line, data, size);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen i_stream_skip(input, size);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen from_line_size -= size;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (from_line_size == 0)
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen break;
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen if (i_stream_read(input) < 0)
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen return -1;
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen }
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen return 0;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen}
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainenstatic int
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainenmbox_write_from_line(struct mbox_sync_mail_context *ctx, off_t move_diff)
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen{
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen string_t *str = ctx->sync_ctx->from_line;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen if (pwrite_full(ctx->sync_ctx->fd, str_data(str), str_len(str),
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen ctx->from_offset + move_diff) < 0) {
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen mbox_set_syscall_error(ctx->sync_ctx->ibox, "pwrite_full()");
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen return -1;
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen }
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen istream_raw_mbox_flush(ctx->sync_ctx->input);
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen return 0;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen}
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainenstatic void
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainenupdate_from_offsets(struct index_mailbox *ibox,
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen struct mail_index_transaction *t, buffer_t *mails_buf,
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen uint32_t seq1, uint32_t seq2)
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen{
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen const struct mbox_sync_mail *mails;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen uint32_t extra_idx = ibox->mbox_extra_idx;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen uint64_t offset;
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen
ecdce39e5ef4b62eefa9f5818f17d153fd5d710aTimo Sirainen mails = buffer_get_modifyable_data(mails_buf, NULL);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen for (; seq1 <= seq2; seq1++, mails++) {
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen if (mails->uid != 0) {
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen offset = mails->from_offset;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen mail_index_update_extra_rec(t, seq1, extra_idx,
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen &offset);
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen }
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen }
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen}
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainenstatic int mbox_sync_check_excl_lock(struct mbox_sync_context *sync_ctx)
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen{
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen int ret;
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen if (sync_ctx->ibox->mbox_lock_type == F_RDLCK) {
3852872e6954b7132e637294132005e86b8ebd4aTimo Sirainen if ((ret = mbox_sync_lock(sync_ctx, F_WRLCK)) < 0)
4cf5f0934a25f1fd58f2780108f9d6498c455a1fTimo Sirainen return -1;
4cf5f0934a25f1fd58f2780108f9d6498c455a1fTimo Sirainen if (ret == 0)
4cf5f0934a25f1fd58f2780108f9d6498c455a1fTimo Sirainen return -2;
4cf5f0934a25f1fd58f2780108f9d6498c455a1fTimo Sirainen }
4cf5f0934a25f1fd58f2780108f9d6498c455a1fTimo Sirainen return 0;
4cf5f0934a25f1fd58f2780108f9d6498c455a1fTimo Sirainen}
4cf5f0934a25f1fd58f2780108f9d6498c455a1fTimo Sirainen
4cf5f0934a25f1fd58f2780108f9d6498c455a1fTimo Sirainenstatic int mbox_sync_handle_expunge(struct mbox_sync_mail_context *mail_ctx)
4cf5f0934a25f1fd58f2780108f9d6498c455a1fTimo Sirainen{
4cf5f0934a25f1fd58f2780108f9d6498c455a1fTimo Sirainen int ret;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen if ((ret = mbox_sync_check_excl_lock(mail_ctx->sync_ctx)) < 0)
4ba9a1d3facc515b3feb5238a16bcf91f76fac61Timo Sirainen return ret;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen mail_ctx->mail.offset = mail_ctx->from_offset;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen mail_ctx->mail.space =
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen mail_ctx->body_offset - mail_ctx->from_offset +
563273bdac80393af63b9520cbf4d24cc0efd028Timo Sirainen mail_ctx->mail.body_size;
c6afd726060aae56b6622c6c52aec10231c4bf1cTimo Sirainen mail_ctx->mail.body_size = 0;
047e3bbb00e68a0d43355e11a67b2e912e06de19Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen if (mail_ctx->sync_ctx->dest_first_mail) {
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen /* expunging first message, fix space to contain next
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen message's \n header too since it will be removed. */
34f4c7610b846a945779b6be78d1ef575c7d0ca8Timo Sirainen mail_ctx->mail.space++;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen }
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen mail_ctx->sync_ctx->expunged_space += mail_ctx->mail.space;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen return 0;
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen}
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
db0735f9b388c5bcfb781b1b25015e898d63d953Timo Sirainenstatic int mbox_sync_handle_header(struct mbox_sync_mail_context *mail_ctx)
047e3bbb00e68a0d43355e11a67b2e912e06de19Timo Sirainen{
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen off_t move_diff;
eca30f1fe8556c46abc75c94d03f59b2e89d4162Timo Sirainen int ret;
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen
17ad2164c747cedbf81dae1893063e71a3df0356Timo Sirainen if (sync_ctx->ibox->mbox_readonly)
61b0637759146621cbb7edcbd0b03a71cfd66dfeTimo Sirainen return 0;
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen
421d30619384e72a27e2a5d13ff6525aff4d17feTimo Sirainen if (sync_ctx->expunged_space > 0 && sync_ctx->need_space_seq == 0) {
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen /* move the header backwards to fill expunged space */
6469cf211a57433335641725dc236ebb2b9fdd3bTimo Sirainen if ((ret = mbox_sync_check_excl_lock(sync_ctx)) < 0)
62041dfb7d6ac6e9c633a557075999cdfcff7bd5Timo Sirainen return ret;
62041dfb7d6ac6e9c633a557075999cdfcff7bd5Timo Sirainen
62041dfb7d6ac6e9c633a557075999cdfcff7bd5Timo Sirainen move_diff = -sync_ctx->expunged_space;
62041dfb7d6ac6e9c633a557075999cdfcff7bd5Timo Sirainen
62041dfb7d6ac6e9c633a557075999cdfcff7bd5Timo Sirainen /* read the From-line before rewriting overwrites it */
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if (mbox_read_from_line(mail_ctx) < 0)
fbd918f47f591f8084fd52b207ef29515ddd11b9Timo Sirainen return -1;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen mbox_sync_update_header(mail_ctx, sync_ctx->syncs);
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if ((ret = mbox_sync_try_rewrite(mail_ctx, move_diff)) < 0)
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen return -1;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
eb0816090cf5a549280ad783b9aa6fec199d36baTimo Sirainen if (ret > 0) {
eb0816090cf5a549280ad783b9aa6fec199d36baTimo Sirainen /* rewrite successful, write From-line to
eb0816090cf5a549280ad783b9aa6fec199d36baTimo Sirainen new location */
8a13b020f90e080570658b18c042e7e352c8b14fTimo Sirainen mail_ctx->mail.from_offset += move_diff;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen mail_ctx->mail.offset += move_diff;
f5a7396b31762a1f876517e13ce9065820139f7cTimo Sirainen if (mbox_write_from_line(mail_ctx, move_diff) < 0)
f5a7396b31762a1f876517e13ce9065820139f7cTimo Sirainen return -1;
f5a7396b31762a1f876517e13ce9065820139f7cTimo Sirainen }
f5a7396b31762a1f876517e13ce9065820139f7cTimo Sirainen } else if (mail_ctx->need_rewrite ||
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen buffer_get_used_size(sync_ctx->syncs) != 0) {
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen if ((ret = mbox_sync_check_excl_lock(sync_ctx)) < 0)
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen return ret;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen mbox_sync_update_header(mail_ctx, sync_ctx->syncs);
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen if ((ret = mbox_sync_try_rewrite(mail_ctx, 0)) < 0)
1460ef7a18c53216ddb4a94bb62fba96076aae8eTimo Sirainen return -1;
1460ef7a18c53216ddb4a94bb62fba96076aae8eTimo Sirainen } else {
1460ef7a18c53216ddb4a94bb62fba96076aae8eTimo Sirainen /* nothing to do */
4cf5f0934a25f1fd58f2780108f9d6498c455a1fTimo Sirainen return 0;
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen }
24e5e4526d8f5cbc056ab97fd0d154d0936d7a5eTimo Sirainen
499fec3443374cc89fb8c83b8027c1614097d7a3Timo Sirainen if (ret == 0 && sync_ctx->need_space_seq == 0) {
499fec3443374cc89fb8c83b8027c1614097d7a3Timo Sirainen /* first mail with no space to write it */
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen sync_ctx->need_space_seq = sync_ctx->seq;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen sync_ctx->space_diff = 0;
8d587838c414c48a331f0b54cd7ffd97e5024abdTimo Sirainen
499fec3443374cc89fb8c83b8027c1614097d7a3Timo Sirainen if (sync_ctx->expunged_space > 0) {
/* create dummy message to describe the expunged data */
struct mbox_sync_mail mail;
memset(&mail, 0, sizeof(mail));
mail.offset = mail_ctx->from_offset -
sync_ctx->expunged_space;
mail.space = sync_ctx->expunged_space;
sync_ctx->need_space_seq--;
buffer_append(sync_ctx->mails, &mail, sizeof(mail));
}
}
return 0;
}
static int
mbox_sync_handle_missing_space(struct mbox_sync_mail_context *mail_ctx)
{
struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
uoff_t extra_space;
buffer_append(sync_ctx->mails, &mail_ctx->mail, sizeof(mail_ctx->mail));
sync_ctx->space_diff += mail_ctx->mail.space;
if (sync_ctx->space_diff < 0)
return 0;
/* we have enough space now */
extra_space = MBOX_HEADER_EXTRA_SPACE *
(sync_ctx->seq - sync_ctx->need_space_seq + 1);
if (mail_ctx->mail.uid == 0 &&
(uoff_t)sync_ctx->space_diff > extra_space) {
/* don't waste too much on extra spacing */
sync_ctx->expunged_space = sync_ctx->space_diff - extra_space;
sync_ctx->space_diff = extra_space;
} else {
sync_ctx->expunged_space = 0;
}
if (mbox_sync_rewrite(sync_ctx, sync_ctx->space_diff,
sync_ctx->need_space_seq, sync_ctx->seq) < 0)
return -1;
update_from_offsets(sync_ctx->ibox, sync_ctx->t, sync_ctx->mails,
sync_ctx->need_space_seq, sync_ctx->seq);
/* mail_ctx may contain wrong data after rewrite, so make sure we
don't try to access it */
memset(mail_ctx, 0, sizeof(*mail_ctx));
sync_ctx->need_space_seq = 0;
buffer_set_used_size(sync_ctx->mails, 0);
return 0;
}
static int
mbox_sync_seek_to_uid(struct mbox_sync_context *sync_ctx, uint32_t uid)
{
uint32_t seq;
uint64_t offset;
if (mail_index_lookup_uid_range(sync_ctx->sync_view, uid, uid,
&seq, &seq) < 0)
return -1;
if (seq == 0)
return 0;
if (mbox_sync_get_from_offset(sync_ctx, seq, &offset) < 0)
return -1;
/* set to -1, since they're always increased later */
sync_ctx->seq = sync_ctx->idx_seq = seq-1;
sync_ctx->dest_first_mail = sync_ctx->seq == 0;
if (istream_raw_mbox_seek(sync_ctx->input, offset) < 0) {
mail_storage_set_critical(sync_ctx->ibox->box.storage,
"Cached message offset %s is invalid for mbox file %s",
dec2str(offset), sync_ctx->ibox->path);
mail_index_mark_corrupted(sync_ctx->ibox->index);
return -1;
}
(void)istream_raw_mbox_get_body_offset(sync_ctx->input);
return 0;
}
static int mbox_sync_loop(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail_context *mail_ctx,
uint32_t min_message_count)
{
const struct mail_index_record *rec;
uint32_t uid, messages_count;
uoff_t offset;
int ret, expunged;
if (min_message_count != 0) {
if (istream_raw_mbox_seek(sync_ctx->input, 0) < 0) {
/* doesn't begin with a From-line */
mail_storage_set_error(sync_ctx->ibox->box.storage,
"Mailbox isn't a valid mbox file");
return -1;
}
sync_ctx->dest_first_mail = TRUE;
} else {
/* we sync only what we need to. jump to first record that
needs updating */
const struct mail_index_sync_rec *sync_rec;
size_t size;
if (buffer_get_used_size(sync_ctx->syncs) == 0 &&
sync_ctx->sync_rec.uid1 == 0) {
if (mbox_sync_read_index_syncs(sync_ctx, 1,
&expunged) < 0)
return -1;
if (buffer_get_used_size(sync_ctx->syncs) == 0 &&
sync_ctx->sync_rec.uid1 == 0) {
/* nothing to do */
return 0;
}
}
sync_rec = buffer_get_data(sync_ctx->syncs, &size);
if (size == 0)
sync_rec = &sync_ctx->sync_rec;
if (mbox_sync_seek_to_uid(sync_ctx, sync_rec->uid1) < 0)
return -1;
}
while ((ret = mbox_sync_read_next_mail(sync_ctx, mail_ctx)) > 0) {
uid = mail_ctx->mail.uid;
/* get all sync records related to this message */
if (mbox_sync_read_index_syncs(sync_ctx, uid, &expunged) < 0)
return -1;
if (!expunged) {
ret = mbox_sync_handle_header(mail_ctx);
sync_ctx->dest_first_mail = FALSE;
} else {
mail_ctx->mail.uid = 0;
ret = mbox_sync_handle_expunge(mail_ctx);
}
if (ret < 0) {
/* -1 = error, -2 = need to restart */
return ret;
}
if (mbox_sync_read_index_rec(sync_ctx, uid, &rec) < 0)
return -1;
if (!expunged) {
if (mbox_sync_update_index(sync_ctx, &mail_ctx->mail,
rec) < 0)
return -1;
}
istream_raw_mbox_next(sync_ctx->input,
mail_ctx->mail.body_size);
offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
if (sync_ctx->need_space_seq != 0) {
if (mbox_sync_handle_missing_space(mail_ctx) < 0)
return -1;
if (mbox_sync_seek(sync_ctx, offset) < 0)
return -1;
} else if (sync_ctx->expunged_space > 0) {
if (!expunged) {
/* move the body */
if (mbox_move(sync_ctx,
mail_ctx->body_offset -
sync_ctx->expunged_space,
mail_ctx->body_offset,
mail_ctx->mail.body_size) < 0)
return -1;
if (mbox_sync_seek(sync_ctx, offset) < 0)
return -1;
}
} else if (sync_ctx->seq >= min_message_count) {
mbox_sync_buffer_delete_old(sync_ctx->syncs, uid+1);
if (buffer_get_used_size(sync_ctx->syncs) == 0) {
/* if there's no sync records left,
we can stop */
if (sync_ctx->sync_rec.uid1 == 0)
break;
/* we can skip forward to next record which
needs updating. */
uid = sync_ctx->sync_rec.uid1;
if (mbox_sync_seek_to_uid(sync_ctx, uid) < 0)
return -1;
}
}
}
if (istream_raw_mbox_is_eof(sync_ctx->input)) {
/* rest of the messages in index don't exist -> expunge them */
messages_count =
mail_index_view_get_message_count(sync_ctx->sync_view);
while (sync_ctx->idx_seq < messages_count)
mail_index_expunge(sync_ctx->t, ++sync_ctx->idx_seq);
}
return 0;
}
static int mbox_sync_handle_eof_updates(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail_context *mail_ctx)
{
uoff_t offset, extra_space, trailer_size;
if (!istream_raw_mbox_is_eof(sync_ctx->input)) {
i_assert(sync_ctx->need_space_seq == 0);
i_assert(sync_ctx->expunged_space == 0);
return 0;
}
trailer_size = i_stream_get_size(sync_ctx->file_input) -
sync_ctx->file_input->v_offset;
if (sync_ctx->need_space_seq != 0) {
i_assert(sync_ctx->space_diff < 0);
extra_space = MBOX_HEADER_EXTRA_SPACE *
(sync_ctx->seq - sync_ctx->need_space_seq + 1);
sync_ctx->space_diff -= extra_space;
sync_ctx->space_diff += sync_ctx->expunged_space;
if (sync_ctx->expunged_space <= -sync_ctx->space_diff)
sync_ctx->expunged_space = 0;
else
sync_ctx->expunged_space -= -sync_ctx->space_diff;
if (mail_ctx->have_eoh && !mail_ctx->updated)
str_append_c(mail_ctx->header, '\n');
if (sync_ctx->space_diff < 0 &&
mbox_sync_grow_file(sync_ctx, mail_ctx,
-sync_ctx->space_diff) < 0)
return -1;
if (mbox_sync_try_rewrite(mail_ctx, 0) < 0)
return -1;
if (sync_ctx->seq != sync_ctx->need_space_seq) {
buffer_set_used_size(sync_ctx->mails,
(sync_ctx->seq -
sync_ctx->need_space_seq) *
sizeof(mail_ctx->mail));
buffer_append(sync_ctx->mails, &mail_ctx->mail,
sizeof(mail_ctx->mail));
if (mbox_sync_rewrite(sync_ctx, extra_space,
sync_ctx->need_space_seq,
sync_ctx->seq) < 0)
return -1;
update_from_offsets(sync_ctx->ibox, sync_ctx->t,
sync_ctx->mails,
sync_ctx->need_space_seq,
sync_ctx->seq);
}
sync_ctx->need_space_seq = 0;
buffer_set_used_size(sync_ctx->mails, 0);
}
if (sync_ctx->expunged_space > 0) {
/* copy trailer, then truncate the file */
offset = i_stream_get_size(sync_ctx->file_input) -
sync_ctx->expunged_space - trailer_size;
if (mbox_move(sync_ctx, offset,
offset + sync_ctx->expunged_space,
trailer_size) < 0)
return -1;
if (ftruncate(sync_ctx->fd, offset + trailer_size) < 0) {
mbox_set_syscall_error(sync_ctx->ibox, "ftruncate()");
return -1;
}
sync_ctx->expunged_space = 0;
istream_raw_mbox_flush(sync_ctx->input);
}
return 0;
}
static int mbox_sync_update_index_header(struct mbox_sync_context *sync_ctx)
{
struct stat st;
if (fstat(sync_ctx->fd, &st) < 0) {
mbox_set_syscall_error(sync_ctx->ibox, "fstat()");
return -1;
}
if ((sync_ctx->base_uid_validity != 0 &&
sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) ||
(sync_ctx->hdr->uid_validity == 0 && sync_ctx->seen_first_mail)) {
if (sync_ctx->base_uid_validity == 0) {
/* we couldn't rewrite X-IMAPbase because it's
a read-only mbox */
i_assert(sync_ctx->ibox->mbox_readonly);
sync_ctx->base_uid_validity = time(NULL);
}
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, uid_validity),
&sync_ctx->base_uid_validity,
sizeof(sync_ctx->base_uid_validity));
}
if (istream_raw_mbox_is_eof(sync_ctx->input) &&
sync_ctx->next_uid != sync_ctx->hdr->next_uid) {
i_assert(sync_ctx->next_uid != 0);
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, next_uid),
&sync_ctx->next_uid, sizeof(sync_ctx->next_uid));
}
if ((uint32_t)st.st_mtime != sync_ctx->hdr->sync_stamp) {
uint32_t sync_stamp = st.st_mtime;
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, sync_stamp),
&sync_stamp, sizeof(sync_stamp));
}
if ((uint64_t)st.st_size != sync_ctx->hdr->sync_size) {
uint64_t sync_size = st.st_size;
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, sync_size),
&sync_size, sizeof(sync_size));
}
return 0;
}
static void mbox_sync_restart(struct mbox_sync_context *sync_ctx)
{
sync_ctx->base_uid_validity = 0;
sync_ctx->base_uid_last = 0;
sync_ctx->next_uid = 1;
sync_ctx->prev_msg_uid = 0;
sync_ctx->seq = sync_ctx->idx_seq = 0;
sync_ctx->dest_first_mail = TRUE;
sync_ctx->seen_first_mail = FALSE;
}
static int mbox_sync_do(struct mbox_sync_context *sync_ctx)
{
struct mbox_sync_mail_context mail_ctx;
struct stat st;
uint32_t min_msg_count;
int ret;
if (fstat(sync_ctx->fd, &st) < 0) {
mbox_set_syscall_error(sync_ctx->ibox, "stat()");
return -1;
}
min_msg_count =
(uint32_t)st.st_mtime == sync_ctx->hdr->sync_stamp &&
(uint64_t)st.st_size == sync_ctx->hdr->sync_size ?
0 : (uint32_t)-1;
mbox_sync_restart(sync_ctx);
if ((ret = mbox_sync_loop(sync_ctx, &mail_ctx, min_msg_count)) == -1)
return -1;
if (ret == -2) {
/* initially we had mbox read-locked, but later we needed a
write-lock. doing it required dropping the read lock.
we're here because mbox was modified before we got the
write-lock. so, restart the whole syncing. */
i_assert(sync_ctx->ibox->mbox_lock_type == F_WRLCK);
mail_index_transaction_rollback(sync_ctx->t);
sync_ctx->t = mail_index_transaction_begin(sync_ctx->sync_view,
FALSE);
mbox_sync_restart(sync_ctx);
if (mbox_sync_loop(sync_ctx, &mail_ctx, (uint32_t)-1) < 0)
return -1;
}
if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0)
return -1;
/* only syncs left should be just appends (and their updates)
which weren't synced yet for some reason (crash). we'll just
ignore them, as we've overwritten them above. */
buffer_set_used_size(sync_ctx->syncs, 0);
memset(&sync_ctx->sync_rec, 0, sizeof(sync_ctx->sync_rec));
if (mbox_sync_update_index_header(sync_ctx) < 0)
return -1;
return 0;
}
static int mbox_sync_has_changed(struct index_mailbox *ibox)
{
const struct mail_index_header *hdr;
struct stat st;
if (mail_index_get_header(ibox->view, &hdr) < 0) {
mail_storage_set_index_error(ibox);
return -1;
}
if (stat(ibox->path, &st) < 0) {
mbox_set_syscall_error(ibox, "stat()");
return -1;
}
return (uint32_t)st.st_mtime != hdr->sync_stamp ||
(uint64_t)st.st_size != hdr->sync_size;
}
static int mbox_sync_update_imap_base(struct mbox_sync_context *sync_ctx)
{
struct mbox_sync_mail_context mail_ctx;
if (mbox_sync_seek(sync_ctx, 0) < 0)
return -1;
sync_ctx->t = mail_index_transaction_begin(sync_ctx->sync_view, FALSE);
sync_ctx->update_base_uid_last = sync_ctx->next_uid-1;
if (mbox_sync_check_excl_lock(sync_ctx) == -1)
return -1;
mbox_sync_restart(sync_ctx);
if (mbox_sync_loop(sync_ctx, &mail_ctx, 1) < 0)
return -1;
if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0)
return -1;
if (mbox_sync_update_index_header(sync_ctx) < 0)
return -1;
return 0;
}
int mbox_sync(struct index_mailbox *ibox, int last_commit, int lock)
{
struct mail_index_sync_ctx *index_sync_ctx;
struct mail_index_view *sync_view;
struct mbox_sync_context sync_ctx;
uint32_t seq;
uoff_t offset;
unsigned int lock_id = 0;
int ret, lock_type;
if (lock) {
if (mbox_lock(ibox, F_RDLCK, &lock_id) <= 0)
return -1;
}
if ((ret = mbox_sync_has_changed(ibox)) < 0) {
if (lock)
(void)mbox_unlock(ibox, lock_id);
return -1;
}
if (ret == 0 && !last_commit)
return 0;
if (last_commit) {
seq = ibox->commit_log_file_seq;
offset = ibox->commit_log_file_offset;
} else {
seq = (uint32_t)-1;
offset = (uoff_t)-1;
}
ret = mail_index_sync_begin(ibox->index, &index_sync_ctx, &sync_view,
seq, offset);
if (ret <= 0) {
if (ret < 0)
mail_storage_set_index_error(ibox);
return ret;
}
memset(&sync_ctx, 0, sizeof(sync_ctx));
sync_ctx.ibox = ibox;
sync_ctx.from_line = str_new(default_pool, 256);
sync_ctx.header = str_new(default_pool, 4096);
sync_ctx.index_sync_ctx = index_sync_ctx;
sync_ctx.sync_view = sync_view;
sync_ctx.t = mail_index_transaction_begin(sync_view, FALSE);
sync_ctx.mails = buffer_create_dynamic(default_pool, 4096, (size_t)-1);
sync_ctx.syncs = buffer_create_dynamic(default_pool, 256, (size_t)-1);
ret = mail_index_get_header(sync_view, &sync_ctx.hdr);
i_assert(ret == 0);
lock_type = mail_index_sync_have_more(index_sync_ctx) ?
F_WRLCK : F_RDLCK;
if (lock_type == F_WRLCK && lock) {
(void)mbox_unlock(ibox, lock_id);
lock_id = 0;
}
if (mbox_sync_lock(&sync_ctx, lock_type) < 0)
return -1;
if (mbox_sync_do(&sync_ctx) < 0)
ret = -1;
if (ret < 0)
mail_index_transaction_rollback(sync_ctx.t);
else if (mail_index_transaction_commit(sync_ctx.t, &seq, &offset) < 0)
ret = -1;
else {
ibox->commit_log_file_seq = 0;
ibox->commit_log_file_offset = 0;
}
sync_ctx.t = NULL;
if (mail_index_sync_end(index_sync_ctx) < 0)
ret = -1;
if (sync_ctx.base_uid_last != sync_ctx.next_uid-1 && ret == 0 &&
!ibox->mbox_readonly) {
/* rewrite X-IMAPbase header. do it after mail_index_sync_end()
so previous transactions have been committed. */
/* FIXME: ugly .. */
ret = mail_index_sync_begin(ibox->index,
&sync_ctx.index_sync_ctx,
&sync_ctx.sync_view,
(uint32_t)-1, (uoff_t)-1);
if (ret < 0)
mail_storage_set_index_error(ibox);
else {
ret = mail_index_get_header(sync_ctx.sync_view,
&sync_ctx.hdr);
if ((ret = mbox_sync_update_imap_base(&sync_ctx)) < 0)
mail_index_transaction_rollback(sync_ctx.t);
else if (mail_index_transaction_commit(sync_ctx.t,
&seq,
&offset) < 0)
ret = -1;
if (mail_index_sync_end(sync_ctx.index_sync_ctx) < 0)
ret = -1;
}
}
if (sync_ctx.lock_id != 0 && (ret < 0 || !lock)) {
/* FIXME: drop to read locking and keep it MBOX_SYNC_SECS+1
to make sure we notice changes made by others */
if (mbox_unlock(ibox, sync_ctx.lock_id) < 0)
ret = -1;
}
str_free(sync_ctx.header);
str_free(sync_ctx.from_line);
buffer_free(sync_ctx.mails);
buffer_free(sync_ctx.syncs);
return ret;
}
int mbox_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags)
{
struct index_mailbox *ibox = (struct index_mailbox *)box;
if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 ||
ibox->sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <= ioloop_time) {
ibox->sync_last_check = ioloop_time;
if (mbox_sync(ibox, FALSE, FALSE) < 0)
return -1;
}
return index_storage_sync(box, flags);
}